#!/usr/bin/python3
# 2016.11.18   使用Popen进行多线程
import os
import multiprocessing
import fileinput
import re
import datetime


def dividFlags(flag, thread):                         # 用来分割的函数flags的函数
    if thread == 1:
        return flag
    elif thread > 1:
        print("分配任务中")
        results = []
        size = len(flag)//thread

        for i in range(thread):
            if i == (thread - 1):
                results.append(flag[i * size:])
            else:
                results.append(flag[i * size: size * (i+1)])
        return results


class realDivide(multiprocessing.Process):                      #
    def __init__(self, file, flags, output, num):
        multiprocessing.Process.__init__(self)                    # 这一行很重要，真正继承进程的生成方式
        self.file = file
        self.flags = flags
        self.output = output
        self.num = num

    def run(self):                                               # run(self)是固定搭配，内部变量的赋值是通过self进行的
        flags = self.flags
        file = self.file
        output = self.output
        num = self.num
        with fileinput.input(file) as f:
            old = None
            have = []
            for line in f:
                lines = re.split("\\s+", line)
                flag = lines[num]

                if flag in flags:
                    if flag != old:
                        print("处理：%s" % flag)
                    outputfile = output + "/" + flag

                    if flag not in have:                               # 用have来确定，是不是第一次遇到这个flag
                        if os.path.isfile(outputfile):
                            os.remove(outputfile)
                        w = open(outputfile, 'w')                       # 因为如果不在里边，我就会添加的
                        w.write(line)                                   # 以此来决定是否覆盖源文件
                        w.close()
                        have.append(flag)
                        # if len(have) == len(flags):                     # 如果have和flags的大小一致了，那么也就没必要输出了
                        #     exit()                                      # 如果源文件是按循序来的话，然而不敢保证
                    else:
                        if os.path.isfile(outputfile):
                            w = open(outputfile, 'a')                       # 因为如果不在里边，我就会添加的
                            w.write(line)                                       # 以此来决定是否覆盖源文件
                            w.close()

                    old = flag


def main(file, output, num, thread):
    if not num or int(num) <= 0:  # 针对输入的结果进行处理
        num = 0
    else:
        num = int(num) - 1

    if not thread or int(thread) <= 0:  # 针对输入的结果进行处理
        thread = 1
    else:
        thread = int(thread)
        if thread <= 0:
            thread = 1

    begin = datetime.datetime.now()
    print("程序开始于：%s" % begin)

    output = re.split("'/|\\\$'", output)[0]
    if not os.path.isdir(output):                         # 判断输出目录是否存在，不存在就新建
        os.mkdir(output)

    if thread > 1:
        print("使用%d个进程" % thread)
        flag = []
        proc = []

        print("读取第一遍")
        with fileinput.input(file) as f:
            for line in f:
                lines = re.split("\\s+", line)
                if num > len(lines):
                    print("输入的列大于总列数")
                    exit()
                lines = lines[num]
                if lines not in flag:                      # 第一遍读取，首先把该读取的flags读取完成
                    flag.append(lines)
        print("第一遍读取完成")
        flags = dividFlags(flag, thread)
        # print(len(flags))
        for i in range(thread):
            if not flags[i]:
                print("flags[%d]为空" % i)
                exit()
            proc.append(realDivide(flags=flags[i], file=file, output=output, num=num))

        for i in range(thread):
            p = proc[i]
            p.start()

        for i in iter(proc):
            i.join()

    elif thread == 1:                                      # 单进程的处理
        if os.path.isfile(file):                            # 判断输入是否是一个文件
            flags = []
            with fileinput.input(file) as f:                 # 逐行读取
                have = []
                for line in f:
                    flag = re.split("\\s+", line)[num]      # 根据那个数据分割
                    if flag in flags:
                        if flag != old:
                            print("处理：%s" % flag)
                        outputfile = output + "/" + flag

                        if flag not in have:  # 用have来确定，是不是第一次遇到这个flag
                            if os.path.isfile(outputfile):
                                os.remove(outputfile)
                            w = open(outputfile, 'w')  # 因为如果不在里边，我就会添加的
                            w.write(line)  # 以此来决定是否覆盖源文件
                            w.close()
                            have.append(flag)
                        else:
                            if os.path.isfile(outputfile):
                                w = open(outputfile, 'a')
                                w.write(line)
                                w.close()

                        old = flag
        else:
            print("输入的不是文件")
            exit()

    end = datetime.datetime.now()
    print("程序终止于：%s" % end)
    print("一共花费：%s" % str(end - begin))


if __name__ == "__main__":
    file = input("要分割的文件：")
    output = input("输出目录：")
    num = input("需要分割的列：")
    thread = input("需要使用的进程数：")

    main(file=file, output=output, num=num, thread=thread)
    input("任意键退出")
